#!/usr/bin/env python
import os,sys,pprint,optparse
import urlparse,csv,difflib,time,sets
import math
from Tkinter import *
import FileDialog,tkFileDialog

class ClusterInfo:
    version = '0.1'
    def GetCsvReader(self,file):
        reader = csv.reader(open(file,'rb'))
        return reader
    def print_time_left(self,i,timeleft):
        print 'i: ',i,'time left: ',timeleft
    def getDictFileName(self,fileName):
        return os.path.splitext(fileName)[0]+'.dict'
    def ParseFile(self,fileName,verbose = True):
        dictFileName = self.getDictFileName(fileName)
        self.fileName = fileName
        dictExists = True
        if not os.path.exists(dictFileName):
            dictExists = False
        else:
            dictMtime = os.path.getmtime(dictFileName)
            fileMtime = os.path.getmtime(fileName)
            if dictMtime<fileMtime:
                dictExists = False
        if dictExists:
            # use the dict file
            dictCode = open(dictFileName).read()
            if dictCode[0:11] == "# dict file":
                exec (dictCode)
                self.info = clusterInfo
                # move on
                return clusterInfo
        # create the dict file
        info = {'lines':0,
                'version':ClusterInfo.version,
                'urlTypes':{},
                'urlTypesList':[],
                }
        self.info = info
        if not os.path.exists(fileName):
            return info

        # get number of lines in the file
        print 'Getting number of lines in file'
        f = open(fileName)
        info['lines'] = sum(1 for line in f)
        f.close()

        # get url type info
        reader = self.GetCsvReader(fileName)
        lastime = curtime = time.time()
        urlTypes = []
        for i,row in enumerate(reader):
            if len(row)<12:
                # ignore preliminary lines
                continue
            mod = 10000
            if i%mod==0 and verbose:
                curtime = time.time()
                elapsed = curtime-lastime
                iterleft = info['lines']-i
                timeleft = elapsed*iterleft/mod
                self.print_time_left(i,timeleft)
                lastime = curtime
            urlType = row[0]
            if urlType not in urlTypes:
                urlTypes.append(urlType)
                info['urlTypes'][urlType]=1
            else:
                info['urlTypes'][urlType]+=1
        urlTypesList = info['urlTypes'].items()
        urlTypesList.sort(self.sort_dict_items)
        info['urlTypesList'] = urlTypesList
        self.ToFile()
        return info
    def ToFile(self,dictFileName=None):
        if dictFileName is None:
            dictFileName = self.getDictFileName(self.fileName)
        print 'Writing file:',dictFileName
        f = open(dictFileName,'w')
        f.write('# dict file\n')
        f.write('clusterInfo = \\\n%s\n\n'%(pprint.pformat(self.info)))
        f.close()
    def sort_dict_items(self,x,y):
        if x[1]>y[1]: return 1
        if x[1]<y[1]: return -1
        return 0
            
    def SetupLevels(self,ranges=[100,5000,20000]):
        info = self.info
        info['levels'] = levels = []
        for max in ranges:
            levels.append({'top':max,
                           'max': -1e300,
                           'min': 1e300,
                           'list':[]})
        levels.append({'top':1e300,
                       'max': -1e300,
                       'min': 1e300,
                       'list':[]})
        for i,(urlType,urlNum) in enumerate(info['urlTypesList']):
            for j,level in enumerate(levels):
                if urlNum <= level['top']:
                    if urlNum < level['min']:
                        level['min'] = urlNum
                    if urlNum > level['max']:
                        level['max'] = urlNum
                    level['list'].append({'type':urlType,
                                          'num':urlNum,})
                    break
        for i in range(len(levels)-1,-1,-1):
            if len(levels[i]['list']) == 0:
                levels.pop(i)
        levels.reverse()
        for i,level in enumerate(levels):
            for o in level['list']:
                o['level'] = i
        info['ranges'] = ranges

    def GetClusters(self):
        return self.info

    
class PhishBowlTk:
    def __init__(self,**kw):
        if __name__=='__main__':
            op = optparse.OptionParser()
            op.add_option('-f','--file',dest='fileName',
                          help='Phishing data file to parse',
                          default='aug07.urls.txt')
            op.add_option('-v','--verbose',dest='verbose',
                          default=True)
            op.add_option('--width',dest='initWidth',
                          default=600,type='int')
            op.add_option('--height',dest='initHeight',
                          default=600,type='int')
            (options,args) = op.parse_args()
            self.__dict__.update(options.__dict__)
        self.__dict__.update(kw)

        self.clusterInfo = ClusterInfo()
        self.levelRanges = [100,1000,5000,20000]
        self.OpenClusterFile()
        self.SetupGUI()

    colors = ['red','orange','yellow','green',
              'blue','magenta','violet']
    def get_color(self,index):
        if index>=len(self.colors): index = len(self.colors)-1
        return self.colors[index]
    def cartPolar(self,x,y):
        r = math.sqrt(x**2+y**2)
        theta = math.arccos(x/r)
        if y > 0:
            theta = 2*math.pi - theta
        return r,360*(theta/(2*math.pi))
    def polarCart(self,r,theta):
        return r*math.cos(theta),r*math.sin(theta)

    def Reset(self):
        w,h = map(int,[self.canvas.winfo_width(),
                       self.canvas.winfo_height()])
        if (w,h) == (1,1):
            w,h = map(int,[self.canvas['width'],self.canvas['height']])
        #w,h = map(int,[self.canvas['width'],self.canvas['height']])
        #print w,h
        self.currentLevelSelected = None
        self.currentCluster = None
        self.ResetRangeEntry()
        self.levelListbox.delete(0,'end')
        self.SetupClusters(w,h)

    def OpenClusterFile(self,fileName=None):
        if fileName is not None:
            self.fileName = fileName
        info = self.clusterInfo.ParseFile(self.fileName)
        ranges = info.get('ranges')
        if ranges:
            self.levelRanges = ranges
        self.clusterInfo.SetupLevels(self.levelRanges)
    def getNewClusterFile(self):
        fd = FileDialog.LoadFileDialog(self.mainFrame)
        file = fd.go(pattern="*.txt")
        #file = tkFileDialog.askopenfilename(filetypes=[('CSV (.txt)','txt'),
        #                                               ('All files','*')])
        #print file
        if not file: return
        self.OpenClusterFile(file)
        top = self.mainFrame.winfo_toplevel()
        top.title(file)
        self.Reset()
        
    def SetupLevelDisplay(self,levelNumber,level,bbox,radius=(0,0)):
        clusters = level['list']
        num = len (clusters)
        dtheta = 360/num
        totalUrls = 0 
        for i,cluster in enumerate(clusters):
            num = cluster['num']
            totalUrls += num
        curTheta = 0
        for i,cluster in enumerate(clusters):
            num = float(cluster['num'])
            dtheta = 360*(num/totalUrls)
            color = self.get_color(levelNumber)
            id = self.canvas.create_arc(bbox,start=curTheta,
                                        extent=dtheta,
                                        fill=color,
                                        #width=0,
                                        #style='chord',
                                        )
            if levelNumber < 3:
                cx,cy = (bbox[2]+bbox[0])/2,(bbox[3]+bbox[1])/2.
                rx,ry = radius
                r = rx-.1*cx
                theta = curTheta+(dtheta/2)
                theta = (2*math.pi)*(theta/360.)
                x,y=self.polarCart(r,theta)
                x+=cx;y=cy-y
                tid = self.canvas.create_text(x,y,text=cluster['type'])
                func = lambda e,id=id,s=self:s.HighlightCluster(e,id)
                self.canvas.tag_bind(tid,'<Motion>',func)
                func = lambda e,id=id,s=self:s.SelectCluster(e,id)
                self.canvas.tag_bind(tid,'<Button-1>',func)
                self.clusterIdMap[tid] = None
            cluster['color'] = color
            cluster['id'] = id
            self.clusterIdMap[id] = cluster
            self.canvas.tag_bind(id,'<Motion>',
                                 lambda e,id=id,s=self:s.HighlightCluster(e,id))
            self.canvas.tag_bind(id,'<Button-1>',
                                 lambda e,id=id,s=self:s.SelectCluster(e,id))
            curTheta += dtheta
    clusterIdMap = {}
    def SetupClusters(self,w=600,h=600):
        hw,hh = w/2,h/2
        for id,cluster in self.clusterIdMap.items():
            self.canvas.delete(id)
        clusters = self.clusterInfo.GetClusters()
        numLevels = len(clusters['levels'])
        levels = clusters['levels']
        #levels.reverse()
        self.clusterIdMap = {}
        for i,level in enumerate(clusters['levels']):
            radius = [(numLevels-i)*hw/numLevels-10,
                      (numLevels-i)*hh/numLevels-10]
            bbox = hw-radius[0],hh-radius[1],hw+radius[0],hh+radius[1]
            #print (bbox)
            self.SetupLevelDisplay(i,level,bbox,radius)

    currentLevelSelected = None
    def ListboxSelectCluster(self,event):
        self.levelListbox.update_idletasks()
        sel = self.levelListbox.nearest(event.y)
        levels = self.clusterInfo.GetClusters()['levels']
        items = levels[self.currentLevelSelected]['list']
        cluster = items[sel]
        self.HighlightCluster(None,cluster['id'])
    def SelectCluster(self,event,id):
        self.levelListbox.delete(0,'end')
        levels = self.clusterInfo.GetClusters()['levels']
        cluster = self.clusterIdMap[id]
        levelIndex = cluster['level']
        self.currentLevelSelected = levelIndex
        items = levels[levelIndex]['list']
        index = items.index(cluster)
        ilist = []
        for item in items:
            ilist.append('%s: %s'%(item['num'],item['type'],))
        self.levelListbox.insert(0,*ilist)
        self.levelListbox.selection_set(index)
        self.levelListbox.see(index)
        self.levelScroll.update_idletasks()
        #print levels[]
        
    currentCluster = None
    def HighlightCluster(self,event,id):
        if self.currentCluster:
            cluster = self.clusterIdMap[self.currentCluster]
            self.canvas.itemconfigure(self.currentCluster,
                                      fill=cluster['color'],
                                      )
        self.canvas.itemconfigure(id,fill='white')
        cluster = self.clusterIdMap[id]
        self.nameVariable.set(cluster['type'])
        self.numberVariable.set(cluster['num'])
        self.currentCluster = id

    #-------------------------------------------------
    # Range setting routines
    def SetRanges(self,event):
        print 'Setting Ranges'
        rangeString = self.rangeVariable.get()
        split = rangeString.split(',')
        ranges = []
        for levelString in split:
            levelString = levelString.strip()
            if not levelString: return
            if not levelString.isdigit(): return
            ranges.append(int(levelString))
        ranges.sort()
        # remove duplicates
        ranges = sets.Set(ranges)
        ranges = list(ranges)
        ranges.sort()
        self.levelRanges = ranges
        self.ResetRangeEntry()
        self.clusterInfo.SetupLevels(self.levelRanges)
        self.clusterInfo.ToFile()
        self.Reset()
    def ResetRangeEntry(self):
        self.rangeVariable.set(', '.join(map(str,self.levelRanges)))
    #-------------------------------------------------


    #-------------------------------------------------
    # GUI setup
    def SetupGUI(self):
        w,h = self.initWidth,self.initHeight

        self.mainFrame = main = Frame()
        self.canvas = Canvas(main,width=w,height=h,
                             bd=2,relief='raised',
                             bg='grey',
                             )

        self.infoFrame = Frame(main,
                               padx=2,pady=2,
                               bd=2,relief='raised',
                               )

        targetFrame = Frame(self.infoFrame)
        nameLabel = Label(targetFrame,text='Target:')
        self.nameVariable = StringVar()
        nameEntry = Entry(targetFrame,textvariable=self.nameVariable)
        numberLabel = Label(targetFrame,text='Number:')
        self.numberVariable = StringVar()
        numberEntry = Entry(targetFrame,textvariable=self.numberVariable)
        nameLabel.grid(row=0,column=0)
        nameEntry.grid(row=0,column=1)
        numberLabel.grid(row=1,column=0)
        numberEntry.grid(row=1,column=1)
        targetFrame.grid(row=0,column=0,sticky=E+W+N)

        getNewButton = Button(self.infoFrame,text='Get New Cluster File',
                              command=self.getNewClusterFile)
        getNewButton.grid(row=1,column=0,sticky=E+W+N)

        levelFrame = Frame(self.infoFrame)
        ls = self.levelScroll = Scrollbar(levelFrame,orient=VERTICAL)
        ls.grid(row=0,column=1,sticky=N+S)
        self.levelListbox = Listbox(levelFrame,
                                    yscrollcommand=ls.set)
        self.levelListbox.grid(row=0,column=0,sticky=N+E+W+S)
        self.levelListbox.bind('<Button-1>',self.ListboxSelectCluster)
        ls.config(command = self.levelListbox.yview)
        levelFrame.rowconfigure(0,weight=1)
        levelFrame.grid(row=2,column=0,sticky=N+E+W+S)

        rangeFrame = Frame(self.infoFrame)
        rangeLabel = Label(rangeFrame,text='Ranges:')
        self.rangeVariable = StringVar()
        re = self.rangeEntry = Entry(rangeFrame,
                                     textvariable=self.rangeVariable)
        self.ResetRangeEntry()
        re.bind('<Return>',self.SetRanges)
        rangeLabel.grid(row=0,column=0)
        re.grid(row=0,column=1,sticky=E+W)
        rangeFrame.grid(row=3,column=0,sticky=E+W)
        
        
        self.SetupClusters(w,h)

        self.canvas.grid(row=0,column=0,sticky=N+E+W+S)
        self.infoFrame.rowconfigure(2,weight=1)
        self.infoFrame.grid(row=0,column=1,sticky=N+S)
        main.winfo_toplevel().rowconfigure(0,weight=1)
        main.winfo_toplevel().columnconfigure(0,weight=1)
        main.winfo_toplevel().title(self.fileName)
        main.rowconfigure(0,weight=1)
        main.columnconfigure(0,weight=1)
        main.grid(sticky=N+E+W+S)
        self.canvas.bind('<Configure>',self.Resize)
    #-------------------------------------------------

    def Resize(self,event):
        self.canvas.update_idletasks()
        self.Reset()
        
    def Run(self):
        self.mainFrame.mainloop()

if __name__=='__main__':
    pb = PhishBowlTk()
    pb.Run()
